{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8bb353d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU for acceleration\n",
      "Input data: mnist\n",
      "/workspace/MTL-IBA/MTL-VIBI/compare\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/workspace/MTL-IBA/MTL-VIBI/compare/lib/image_utils.py:43: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  self.img = cuda(torch.tensor(img.squeeze(0), dtype=torch.float), self.is_cuda)\n",
      "/workspace/MTL-IBA/MTL-VIBI/compare/lib/image_utils.py:71: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  seg.view(1, -1)[:, idx_all.type(torch.long)] = cuda(torch.tensor(idx, dtype=torch.float), self.is_cuda)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the number of segment image: 12544\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_9896/949319502.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    471\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    472\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 473\u001b[0;31m     \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    474\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    475\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_9896/949319502.py\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m    126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m     \u001b[0;31m## Prediction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m     \u001b[0mtest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mK\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    130\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_9896/949319502.py\u001b[0m in \u001b[0;36mtest\u001b[0;34m(args, model, device, test_loader, k, **kargs)\u001b[0m\n\u001b[1;32m    243\u001b[0m                                          \u001b[0msegments\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msegments\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m                                          \u001b[0msegments_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msegments_data\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 245\u001b[0;31m                                          chunked = args.chunked)\n\u001b[0m\u001b[1;32m    246\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mexplanation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlocal_exp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mexplanation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtop_labels\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'none'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/workspace/MTL-IBA/MTL-VIBI/compare/lib/lime_library.py\u001b[0m in \u001b[0;36mexplain_instance\u001b[0;34m(self, image, filter_size, classifier_fn, labels, hide_color, top_labels, num_features, num_samples, batch_size, segments, segments_data, distance_metric, model_regressor, random_seed, chunked)\u001b[0m\n\u001b[1;32m    406\u001b[0m             \u001b[0msegments_data\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    407\u001b[0m             \u001b[0mclassifier_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_samples\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 408\u001b[0;31m             batch_size=batch_size)\n\u001b[0m\u001b[1;32m    409\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    410\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/workspace/MTL-IBA/MTL-VIBI/compare/lib/lime_library.py\u001b[0m in \u001b[0;36mdata_labels\u001b[0;34m(self, image, fudged_image, segments, segments_data, classifier_fn, num_samples, batch_size)\u001b[0m\n\u001b[1;32m    482\u001b[0m             \u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msegments\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    483\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mz\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzeros\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 484\u001b[0;31m                 \u001b[0mmask\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msegments\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    485\u001b[0m             \u001b[0mtemp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfudged_image\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    486\u001b[0m             \u001b[0mimgs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import argparse\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]='3'\n",
    "\n",
    "import sys\n",
    "import re\n",
    "import copy\n",
    "import matplotlib.pyplot as plt\n",
    "sys.path.append('../')\n",
    "import numpy as np\n",
    "from return_data import return_data\n",
    "#from scipy import misc\n",
    "#import cv2\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import time\n",
    "from torch.autograd import Variable\n",
    "from utils import cuda, str2bool, label2binary, cuda, idxtobool, UnknownDatasetError, index_transfer, save_batch\n",
    "from pathlib import Path\n",
    "from lib.image_utils import Segmentation\n",
    "from lime.wrappers.scikit_image import SegmentationAlgorithm\n",
    "from torch.nn import functional as F\n",
    "from sklearn.utils import check_random_state\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score, roc_auc_score\n",
    "from sklearn.linear_model import LogisticRegression  # Ridge\n",
    "import matplotlib\n",
    "matplotlib.use('agg')\n",
    "import matplotlib.pyplot as plt\n",
    "matplotlib.pyplot.switch_backend('agg')\n",
    "# from torchvision.models import vggz16, vgg19\n",
    "# from torchvision.utils import save_image\n",
    "# from lib.gradients import VanillaGrad, SmoothGrad, GuidedBackpropGrad, GuidedBackpropSmoothGrad\n",
    "# from lib.image_utils import preprocess_image, save_as_gray_image\n",
    "# from lib.labels import IMAGENET_LABELS\n",
    "try:\n",
    "    import lime1\n",
    "except:\n",
    "    sys.path.append(os.path.join('..', '..'))  # add the current directory\n",
    "    import lime1\n",
    "#from lime import lime_image\n",
    "# from lime.wrappers.scikit_image import SegmentationAlgorithm\n",
    "from lib.lime_library import LimeImageExplainerModified, LimeTextExplainerModified\n",
    "from lib.image_utils import Segmentation\n",
    "\n",
    "def parse_args():\n",
    "\n",
    "    parser = argparse.ArgumentParser()\n",
    "    \n",
    "    parser.add_argument('--dataset', default='mnist', type=str, help='dataset name: imdb-sent, imdb-word, mnist, mimic')\n",
    "    parser.add_argument('--default_dir', default='../mnist/', type=str, help='default directory path')\n",
    "    parser.add_argument('--data_dir', default='../mnist/dataset/Dataset_BUSI_AN/train/images', type=str, help='data directory path')\n",
    "    parser.add_argument('--batch_size', type=int, default=8, metavar='N', help='input batch size for training (default: 64)')\n",
    "    parser.add_argument('--model_name', default='original_BUSI6_40_0.93750.pth', type=str, help= 'if train is True, model name to be saved, otherwise model name to be loaded')\n",
    "    parser.add_argument('--chunk_size', default=1, type=int, help='chunk size. for image, chunk x chunk will be the actual chunk size')\n",
    "    parser.add_argument('--cuda', default=True, type=str2bool, help='enable cuda')\n",
    "    parser.add_argument('--out_dir', type=str, default='./result/lime/', help='Result directory path')\n",
    "    parser.add_argument('--K', type=int, default=-1, help='dimension of encoding Z')\n",
    "    parser.add_argument('--chunked', type=str2bool, default=True, help='True is neighborhood samples are fuzzed within each chunk')\n",
    "    parser.add_argument('--segment_filter_size', type=int, default=1, help='only for imdb, -1 for cnn, 10 for lstm')\n",
    "    args = parser.parse_args(args=[])\n",
    "    \n",
    "    args.cuda = args.cuda and torch.cuda.is_available()\n",
    "    if args.cuda:\n",
    "        print(\"Using GPU for acceleration\")\n",
    "    else:\n",
    "        print(\"Using CPU for computation\")\n",
    "    \n",
    "    print('Input data: {}'.format(args.dataset))\n",
    "\n",
    "    return args\n",
    "\n",
    "def main():\n",
    "\n",
    "    args = parse_args()\n",
    "\n",
    "    if not os.path.exists(args.out_dir):\n",
    "\n",
    "        os.makedirs(args.out_dir)\n",
    "\n",
    "    ## Data Loader\n",
    "    args.root = os.path.join(args.data_dir)\n",
    "    args.load_pred = False\n",
    "    args.device = torch.device(\"cuda\" if args.cuda else \"cpu\")\n",
    "    args.model_dir = args.default_dir + 'models'\n",
    "    device = torch.device(\"cuda\" if args.cuda else \"cpu\")\n",
    "    \n",
    "    data_loader = return_data(args)\n",
    "    test_loader = data_loader['test']\n",
    "\n",
    "    if 'mnist' in args.dataset:\n",
    "    \n",
    "#         from mnist.original import Net\n",
    "        from Net import Net\n",
    "        print(os.getcwd())\n",
    "        ## load model\n",
    "        model = Net().to(device) \n",
    "        \n",
    "        args.word_idx = None\n",
    "        args.original_ncol = 224\n",
    "        args.original_nrow = 224\n",
    "        args.chunk_size = args.chunk_size if args.chunk_size > 0 else 1\n",
    "        assert np.remainder(args.original_nrow, args.chunk_size) == 0\n",
    "        args.filter_size = (args.chunk_size, args.chunk_size)\n",
    "        args.explainer = LimeImageExplainerModified(verbose=False, feature_selection='highest_weights', is_cuda=args.cuda, dataset=args.dataset)\n",
    "        args.explainer_all = LimeImageExplainerModified(verbose=False, feature_selection='none', is_cuda=args.cuda, dataset=args.dataset)\n",
    "        args.segment_filter_size = 2\n",
    "        args.segmenter = Segmentation(dataset=args.dataset,\n",
    "                                      filter_size=(args.segment_filter_size, args.segment_filter_size),\n",
    "                                      is_cuda=args.cuda)\n",
    "        args.model_regressor = LogisticRegression(random_state=0, solver='lbfgs', max_iter=200, fit_intercept=True, multi_class='ovr')\n",
    "        args.num_samples = 64\n",
    "    else:\n",
    "    \n",
    "        raise UnknownDatasetError()\n",
    "\n",
    "#     model_name = Path(args.model_dir).joinpath(args.model_name)\n",
    "    model_name = Path(args.default_dir).joinpath(args.model_name)\n",
    "    model.load_state_dict(torch.load(model_name, map_location='cpu'))\n",
    "\n",
    "    if args.cuda:\n",
    "        \n",
    "        model.cuda()\n",
    "#        model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(mo\n",
    "#        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[0, 1])\n",
    "\n",
    "    ## Prediction\n",
    "    test(args, model, device, test_loader, k=args.K)\n",
    "\n",
    "def test(args, model, device, test_loader, k, **kargs):\n",
    "    '''\n",
    "    k: the number of raw features selected\n",
    "    '''\n",
    "\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    total_num = 0\n",
    "    total_num_ind = 0\n",
    "    correct = 0\n",
    "    \n",
    "    correct_zeropadded = 0\n",
    "    precision_macro_zeropadded = 0  \n",
    "    precision_micro_zeropadded = 0\n",
    "    precision_weighted_zeropadded = 0\n",
    "    recall_macro_zeropadded = 0\n",
    "    recall_micro_zeropadded = 0\n",
    "    recall_weighted_zeropadded = 0\n",
    "    f1_macro_zeropadded = 0\n",
    "    f1_micro_zeropadded = 0\n",
    "    f1_weighted_zeropadded = 0\n",
    "\n",
    "    vmi_zeropadded_sum = 0\n",
    "    vmi_fidel_sum = 0\n",
    "    vmi_fidel_fixed_sum = 0 \n",
    "    \n",
    "    correct_approx = 0\n",
    "    precision_macro_approx = 0\n",
    "    precision_micro_approx = 0\n",
    "    precision_weighted_approx = 0\n",
    "    recall_macro_approx = 0\n",
    "    recall_micro_approx = 0\n",
    "    recall_weighted_approx = 0\n",
    "    f1_macro_approx = 0\n",
    "    f1_micro_approx = 0\n",
    "    f1_weighted_approx = 0\n",
    "    \n",
    "    correct_approx_fixed = 0\n",
    "    precision_macro_approx_fixed = 0\n",
    "    precision_micro_approx_fixed = 0\n",
    "    precision_weighted_approx_fixed = 0\n",
    "    recall_macro_approx_fixed = 0\n",
    "    recall_micro_approx_fixed = 0\n",
    "    recall_weighted_approx_fixed = 0\n",
    "    f1_macro_approx_fixed = 0\n",
    "    f1_micro_approx_fixed = 0\n",
    "    f1_weighted_approx_fixed = 0    \n",
    "        \n",
    "    is_cuda = args.cuda\n",
    "    j = 0\n",
    "\n",
    "    start = time.time()\n",
    "    for idx, batch in enumerate(test_loader):  # (data, target, _, _)\n",
    "        # print('####### idx break ####### you must remove this before running')\n",
    "        # if idx > 3:\n",
    "        #     break\n",
    "        if 'mnist' in args.dataset:\n",
    "            \n",
    "            num_labels = 2\n",
    "            data = batch[0]\n",
    "            target = batch[1]\n",
    "            idx_list = [0, 1, 2, 3, 4, 5]\n",
    "        else:\n",
    "            raise UnknownDatasetError()\n",
    "                   \n",
    "        data, target = data.to(device), target.to(device)\n",
    "\n",
    "        output_all, output2 = model(data)\n",
    "#         prediction = torch.argmax(output_all, dim=1)\n",
    "#         test_loss += F.cross_entropy(prediction, target, reduction='sum').item()\n",
    "        pred = output_all.max(1, keepdim=True)[1]  # get the index of the max log-prob\n",
    "        target = target.max(1, keepdim=True)[1] \n",
    "        \n",
    "        correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "        total_num += 1\n",
    "        # total_num_ind += data.size(0)\n",
    "\n",
    "        if idx == 0:\n",
    "            if 'mnist' in args.dataset:\n",
    "                args.segments = args.segmenter(Variable(data[0:1], requires_grad=False))\n",
    "                n_features = np.unique(args.segments).shape[0]\n",
    "                args.segments_data = check_random_state(None).randint(0, 2, args.num_samples * n_features).reshape((args.num_samples, n_features))\n",
    "                args.segments_data[0, :] = 1\n",
    "\n",
    "        total_num_sub = 0\n",
    "        for i in range(data.size(0)):\n",
    "            # print('i', i)\n",
    "\n",
    "            if 'mnist' in args.dataset:\n",
    "            \n",
    "                # Calculate Gradient\n",
    "                input_ = Variable(data[i:(i+1)], requires_grad=False)\n",
    "\n",
    "                # output, _ = model(input_)\n",
    "                # output = torch.max(output)\n",
    "                explanation = args.explainer.explain_instance(input_, \n",
    "                                                              filter_size=args.filter_size,\n",
    "                                                              classifier_fn=model, top_labels=2,\n",
    "                                                              hide_color=0, num_features=k,\n",
    "                                                              num_samples=args.num_samples,\n",
    "                                                              model_regressor=args.model_regressor,\n",
    "                                                              segments=args.segments,\n",
    "                                                              segments_data=args.segments_data,\n",
    "                                                              chunked=args.chunked)\n",
    "                                                              \n",
    "                explanation_all = args.explainer_all.explain_instance(input_, \n",
    "                                         filter_size = args.filter_size,\n",
    "                                         classifier_fn = model, \n",
    "                                         top_labels = 2, \n",
    "                                         hide_color = 0, \n",
    "                                         #num_features = k,\n",
    "                                         num_samples = args.num_samples, \n",
    "                                         model_regressor = args.model_regressor,\n",
    "                                         segments = args.segments,\n",
    "                                         segments_data = args.segments_data,\n",
    "                                         chunked = args.chunked)\n",
    "\n",
    "                if explanation.local_exp[explanation.top_labels[0]] != 'none':\n",
    "                    total_num_ind += 1\n",
    "                    total_num_sub += 1\n",
    "                    if total_num_sub == 1:\n",
    "                        data_final = data[i:(i+1)]\n",
    "                        target_final = target[i:(i+1)]\n",
    "                        pred_final = pred[i:(i+1)]\n",
    "                        output_all_final = output_all[i:(i+1)]\n",
    "                    else:\n",
    "                        data_final = torch.cat((data_final, data[i:(i+1)]), dim = 0)\n",
    "                        target_final = torch.cat((target_final, target[i:(i+1)]), dim = 0)\n",
    "                        pred_final = torch.cat((pred_final, pred[i:(i+1)]), dim = 0)\n",
    "                        output_all_final = torch.cat((output_all_final, output_all[i:(i+1)]), dim = 0)\n",
    "                        \n",
    "                    approx = torch.Tensor(explanation_all.local_pred_proba).unsqueeze(0)\n",
    "                    approx_fixed = torch.Tensor(explanation.local_pred_proba).unsqueeze(0)\n",
    "                    # approx = explanation_all.top_labels[np.argmax(explanation_all.local_pred_proba, axis = -1)]\n",
    "                    # approx_fixed = explanation.top_labels[np.argmax(explanation.local_pred_proba, axis = -1)]\n",
    "                    index0 = cuda(torch.LongTensor([[x[0] for x in explanation.local_exp[explanation.top_labels[0]]]]).unsqueeze(0), args.cuda)\n",
    "            \n",
    "                    if args.chunk_size > 1:\n",
    "                        index = index_transfer(dataset=args.dataset,\n",
    "                                                 idx=index0,\n",
    "                                                 filter_size = args.filter_size,\n",
    "                                                 original_nrow = args.original_nrow,\n",
    "                                                 original_ncol = args.original_ncol, \n",
    "                                                 is_cuda = args.cuda).output\n",
    "                    else:\n",
    "                        index = index0.squeeze(0)\n",
    "                else:\n",
    "                    j += 1\n",
    "\n",
    "            if explanation.local_exp[explanation.top_labels[0]] != 'none':\n",
    "                #if i == 0:\n",
    "                if total_num_sub == 1:\n",
    "                    approx_all = approx\n",
    "                    approx_fixed_all = approx_fixed\n",
    "                    index_all = index\n",
    "                else:\n",
    "                    approx_all = torch.cat((approx_all, approx), dim=0)\n",
    "                    approx_fixed_all = torch.cat((approx_fixed_all, approx_fixed), dim=0)\n",
    "                    index_all = torch.cat((index_all, index), dim=0)\n",
    "\n",
    "        ##define data_final\n",
    "        if 'mnist' in args.dataset:\n",
    "            data_size = data_final.size()\n",
    "            binary_selected_all = idxtobool(index_all, [data_size[0], data_size[2] * data_size[3]], is_cuda)            \n",
    "            data_zeropadded = torch.addcmul(torch.zeros(1), value=1, tensor1=binary_selected_all.view(data_size).type(torch.FloatTensor), tensor2=data_final.type(torch.FloatTensor), out=None)\n",
    "\n",
    "        else:\n",
    "            raise UnknownDatasetError()\n",
    "\n",
    "        print('Batch {} Time spent is {}'.format(idx, time.time() - start)) \n",
    "        # Post-hoc Accuracy (zero-padded accuracy)\n",
    "        output_zeropadded, _ = model(cuda(data_zeropadded, is_cuda))    \n",
    "        # 转为cpu上计算\n",
    "#         output_zeropadded, pred_final = output_zeropadded.cpu(), pred_final.cpu()\n",
    "        \n",
    "        pred_zeropadded = output_zeropadded.max(1, keepdim=True)[1]  # get the index of the max log-probability\n",
    "        correct_zeropadded += pred_zeropadded.eq(pred_final).sum().item()\n",
    "        \n",
    "        pred_final, pred_zeropadded = pred_final.cpu(), pred_zeropadded.cpu()\n",
    "        precision_macro_zeropadded += precision_score(pred_final, pred_zeropadded, average='macro')\n",
    "        precision_micro_zeropadded += precision_score(pred_final, pred_zeropadded, average='micro')\n",
    "        precision_weighted_zeropadded += precision_score(pred_final, pred_zeropadded, average='weighted')\n",
    "        recall_macro_zeropadded += recall_score(pred_final, pred_zeropadded, average='macro')\n",
    "        recall_micro_zeropadded += recall_score(pred_final, pred_zeropadded, average='micro')\n",
    "        recall_weighted_zeropadded += recall_score(pred_final, pred_zeropadded, average='weighted')\n",
    "        f1_macro_zeropadded += f1_score(pred_final, pred_zeropadded, average='macro')\n",
    "        f1_micro_zeropadded += f1_score(pred_final, pred_zeropadded, average='micro')\n",
    "        f1_weighted_zeropadded += f1_score(pred_final, pred_zeropadded, average='weighted')\n",
    "\n",
    "        ## Variational Mutual Information\n",
    "        vmi = torch.sum(torch.addcmul(torch.zeros(1), value=1,\n",
    "                                      tensor1=torch.exp(output_all_final).type(torch.FloatTensor),\n",
    "                                      tensor2=output_zeropadded.type(torch.FloatTensor) -torch.sum(output_all_final, dim = -1).unsqueeze(-1).expand(output_zeropadded.size()).type(torch.FloatTensor),\n",
    "                                      out=None), dim=-1)\n",
    "        vmi = vmi.sum().item()\n",
    "        vmi_zeropadded_sum += vmi\n",
    "\n",
    "        ## Approximation Fidelity (prediction performance)\n",
    "        pred = pred_final.type(torch.LongTensor)\n",
    "        pred_approx = approx_all.topk(1, dim=-1)[1]\n",
    "        #pred_approx = torch.Tensor(approx_all).type(torch.LongTensor).unsqueeze(-1)\n",
    "        pred_approx_fixed = approx_fixed_all.topk(1, dim = -1)[1]\n",
    "        #pred_approx_fixed = torch.Tensor(approx_fixed_all).type(torch.LongTensor).unsqueeze(-1)\n",
    "\n",
    "        pred_approx_logit = F.softmax(torch.log(approx_all), dim=1)\n",
    "        pred_approx_fixed_logit = F.softmax(torch.log(approx_fixed_all), dim=-1)\n",
    "        correct_approx += pred_approx.eq(pred).sum().item()\n",
    "        precision_macro_approx += precision_score(pred, pred_approx, average='macro')\n",
    "        precision_micro_approx += precision_score(pred, pred_approx, average='micro')\n",
    "        precision_weighted_approx += precision_score(pred, pred_approx, average='weighted')\n",
    "        recall_macro_approx += recall_score(pred, pred_approx, average='macro')\n",
    "        recall_micro_approx += recall_score(pred, pred_approx, average='micro')\n",
    "        recall_weighted_approx += recall_score(pred, pred_approx, average='weighted')\n",
    "        f1_macro_approx += f1_score(pred, pred_approx, average='macro')\n",
    "        f1_micro_approx += f1_score(pred, pred_approx, average='micro')\n",
    "        f1_weighted_approx += f1_score(pred, pred_approx, average='weighted')\n",
    "        \n",
    "        correct_approx_fixed += pred_approx_fixed.eq(pred).sum().item()\n",
    "        precision_macro_approx_fixed += precision_score(pred, pred_approx_fixed, average='macro')\n",
    "        precision_micro_approx_fixed += precision_score(pred, pred_approx_fixed, average='micro')\n",
    "        precision_weighted_approx_fixed += precision_score(pred, pred_approx_fixed, average='weighted')\n",
    "        recall_macro_approx_fixed += recall_score(pred, pred_approx_fixed, average='macro')\n",
    "        recall_micro_approx_fixed += recall_score(pred, pred_approx_fixed, average='micro')\n",
    "        recall_weighted_approx_fixed += recall_score(pred, pred_approx_fixed, average='weighted')\n",
    "        f1_macro_approx_fixed += f1_score(pred, pred_approx_fixed, average='macro')\n",
    "        f1_micro_approx_fixed += f1_score(pred, pred_approx_fixed, average='micro')\n",
    "        f1_weighted_approx_fixed += f1_score(pred, pred_approx_fixed, average='weighted')\n",
    "\n",
    "        vmi = torch.sum(torch.addcmul(torch.zeros(1), value=1,\n",
    "                                      tensor1=torch.exp(output_all_final).type(torch.FloatTensor),\n",
    "                                      tensor2=pred_approx_logit.type(torch.FloatTensor) - torch.logsumexp(output_all_final, dim= 0).unsqueeze(0).expand(pred_approx_logit.size()).type(torch.FloatTensor) + torch.log(torch.tensor(output_all_final.size(0)).type(torch.FloatTensor)),\n",
    "                                      out=None), dim=-1)\n",
    "        vmi_fidel_sum += vmi.sum().item()\n",
    "        vmi = torch.sum(torch.addcmul(torch.zeros(1), value = 1,\n",
    "                                      tensor1 = torch.exp(output_all_final).type(torch.FloatTensor),\n",
    "                                      tensor2 = pred_approx_fixed_logit.type(torch.FloatTensor) - torch.logsumexp(output_all_final, dim = 0).unsqueeze(0).expand(pred_approx_fixed_logit.size()).type(torch.FloatTensor) + torch.log(torch.tensor(output_all_final.size(0)).type(torch.FloatTensor)), out=None), dim = -1)\n",
    "        vmi_fidel_fixed_sum += vmi.sum().item()  \n",
    "\n",
    "        #print(\"SAVED!!!!\")\n",
    "        if idx in idx_list:\n",
    "\n",
    "            # filename\n",
    "            mname = re.sub(r'\\.ckpt$', '', str(args.model_name))\n",
    "            filename = 'figure_lime_' + args.dataset + '_model_' + mname + '_filter_' + str(args.segment_filter_size) +  '_chunk' + str(args.chunk_size) + '_'  + str(k) + '_idx' + str(idx) + '.png'\n",
    "            filename = Path(args.out_dir).joinpath(filename)\n",
    "            index_chunk = index_all\n",
    "            \n",
    "#            if args.chunk_size is not 1:\n",
    "#                \n",
    "#                index_chunk = index_transfer(dataset = args.dataset,\n",
    "#                                             idx = index_chunk, \n",
    "#                                             filter_size = args.filter_size,\n",
    "#                                             original_nrow = args.original_nrow,\n",
    "#                                             original_ncol = args.original_ncol, \n",
    "#                                             is_cuda = args.cuda).output\n",
    "\n",
    "            save_batch(dataset=args.dataset,\n",
    "                             batch=data_final, label=target_final, label_pred=pred_final.squeeze(-1),\n",
    "                             label_approx=pred_approx_fixed.squeeze(-1),\n",
    "                             index=index_chunk,\n",
    "                             filename=filename,\n",
    "                             is_cuda=args.cuda,\n",
    "                             word_idx=args.word_idx).output\n",
    "\n",
    "        print('[summary] Batch {} Time spent is {}'.format(idx, time.time() - start))                   \n",
    "\n",
    "    # Post-hoc Accuracy (zero-padded accuracy)\n",
    "    accuracy_zeropadded = correct_zeropadded/total_num_ind\n",
    "    precision_macro_zeropadded = precision_macro_zeropadded/total_num\n",
    "    precision_micro_zeropadded = precision_micro_zeropadded/total_num\n",
    "    precision_weighted_zeropadded = precision_weighted_zeropadded/total_num\n",
    "    recall_macro_zeropadded = recall_macro_zeropadded/total_num\n",
    "    recall_micro_zeropadded = recall_micro_zeropadded/total_num\n",
    "    recall_weighted_zeropadded = recall_weighted_zeropadded/total_num\n",
    "    f1_macro_zeropadded = f1_macro_zeropadded/total_num\n",
    "    f1_micro_zeropadded = f1_micro_zeropadded/total_num\n",
    "    f1_weighted_zeropadded = f1_weighted_zeropadded/total_num\n",
    "\n",
    "    # VMI\n",
    "    vmi_zeropadded = vmi_zeropadded_sum/total_num_ind\n",
    "    vmi_fidel = vmi_fidel_sum / total_num_ind\n",
    "    vmi_fidel_fixed = vmi_fidel_fixed_sum / total_num_ind\n",
    "    \n",
    "    ## Approximation Fidelity (prediction performance)\n",
    "    accuracy_approx = correct_approx/total_num_ind\n",
    "    precision_macro_approx = precision_macro_approx/total_num\n",
    "    precision_micro_approx = precision_micro_approx/total_num\n",
    "    precision_weighted_approx = precision_weighted_approx/total_num\n",
    "    recall_macro_approx = recall_macro_approx/total_num\n",
    "    recall_micro_approx = recall_micro_approx/total_num\n",
    "    recall_weighted_approx = recall_weighted_approx/total_num\n",
    "    f1_macro_approx = f1_macro_approx/total_num\n",
    "    f1_micro_approx = f1_micro_approx/total_num\n",
    "    f1_weighted_approx = f1_weighted_approx/total_num\n",
    "    \n",
    "    accuracy_approx_fixed = correct_approx_fixed/total_num_ind\n",
    "    precision_macro_approx_fixed = precision_macro_approx_fixed/total_num\n",
    "    precision_micro_approx_fixed = precision_micro_approx_fixed/total_num\n",
    "    precision_weighted_approx_fixed = precision_weighted_approx_fixed/total_num\n",
    "    recall_macro_approx_fixed = recall_macro_approx_fixed/total_num\n",
    "    recall_micro_approx_fixed = recall_micro_approx_fixed/total_num\n",
    "    recall_weighted_approx_fixed = recall_weighted_approx_fixed/total_num\n",
    "    f1_macro_approx_fixed = f1_macro_approx_fixed/total_num\n",
    "    f1_micro_approx_fixed = f1_micro_approx_fixed/total_num\n",
    "    f1_weighted_approx_fixed = f1_weighted_approx/total_num\n",
    "    \n",
    "    ## Approximation Fidelity (prediction performance)\n",
    "\n",
    "    print('\\n\\n[VAL RESULT]\\n')\n",
    "    #tab = pd.crosstab(y_class, prediction)\n",
    "    #print(tab, end = \"\\n\")                \n",
    "    #print('IZY:{:.2f} IZX:{:.2f}'\n",
    "    #        .format(izy_bound.item(), izx_bound.item()), end = '\\n')\n",
    "    print('acc_zeropadded:{:.4f} avg_acc:{:.4f} avg_acc_fixed:{:.4f}'.format(accuracy_zeropadded, accuracy_approx, accuracy_approx_fixed), end = '\\n')    \n",
    "    print('precision_macro_zeropadded:{:.4f} precision_macro_approx:{:.4f} precision_macro_approx_fixed:{:.4f}'\n",
    "            .format(precision_macro_zeropadded, precision_macro_approx, precision_macro_approx_fixed), end = '\\n')   \n",
    "    print('precision_micro_zeropadded:{:.4f} precision_micro_approx:{:.4f} precision_micro_approx_fixed:{:.4f}'\n",
    "            .format(precision_micro_zeropadded, precision_micro_approx, precision_micro_approx_fixed), end = '\\n')   \n",
    "    print('recall_macro_zeropadded:{:.4f} recall_macro_approx:{:.4f} recall_macro_approx_fixed:{:.4f}'\n",
    "            .format(recall_macro_zeropadded, recall_macro_approx, recall_macro_approx_fixed), end = '\\n')   \n",
    "    print('recall_micro_zeropadded:{:.4f} recall_micro_approx:{:.4f} recall_micro_approx_fixed:{:.4f}'\n",
    "            .format(recall_micro_zeropadded, recall_micro_approx, recall_micro_approx_fixed), end = '\\n') \n",
    "    print('f1_macro_zeropadded:{:.4f} f1_macro_approx:{:.4f} f1_macro_approx_fixed:{:.4f}'\n",
    "            .format(f1_macro_zeropadded, f1_macro_approx, f1_macro_approx_fixed), end = '\\n')   \n",
    "    print('f1_micro_zeropadded:{:.4f} f1_micro_approx:{:.4f} f1_micro_approx_fixed:{:.4f}'\n",
    "            .format(f1_micro_zeropadded, f1_micro_approx_fixed, f1_micro_approx_fixed), end = '\\n') \n",
    "    print('vmi:{:.4f} vmi_fixed:{:.4f} vmi_zeropadded:{:.4f}'.format(vmi_fidel, vmi_fidel_fixed, vmi_zeropadded), end = '\\n')\n",
    "    print()\n",
    "    print('samples skipped{} total_num_ind{}'.format(j, total_num_ind))\n",
    "    print(\"[END]\")\n",
    "    \n",
    "#%%\n",
    "#        if outfile:\n",
    "#            \n",
    "#            predictions = np.array(predictions)\n",
    "#            predictions_idx = np.array(predictions_idx)\n",
    "#            inds = predictions_idx.argsort()\n",
    "#            sorted_predictions = predictions[inds]\n",
    "#\n",
    "#            output_name = model_name + '_pred_' + outmode + '.pt'\n",
    "#            torch.save(sorted_predictions, Path(outfile_path).joinpath(output_name))                \n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76dc56e4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
